#!/usr/bin/env python
# -*- coding:utf-8 -*-
import sys
import datetime
import pymongo

import requests
from bs4 import BeautifulSoup
from scpy.logger import get_logger
import time

logger = get_logger(__file__)

reload(sys)
sys.setdefaultencoding("utf-8")
CONN = pymongo.MongoClient('192.168.31.121', 27017)


def get_url_content(url):
    time.sleep(0.5)
    try:
        content = requests.get(url, headers={
            'Cookie': 'global_cookie=wqik28dbtq3seoftvvc7w5v7j1fiap46h9o; SoufunSessionID_Rent=3_1433843464_2233; SoufunSessionID_Esf=3_1433908743_22416; newhouse_user_guid=EDCEC68E-D60B-35AA-32CD-BD8775278374; city=cq; __utma=147393320.1913327650.1433841452.1433995749.1435203276.12; __utmb=147393320.6.10.1435203276; __utmc=147393320; __utmz=147393320.1433989578.10.4.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=fang.com; pinggucitysort=%u91cd%u5e86; pinggucitysortdb=cq; unique_cookie=U_9cy68k3sa5bgxrkkf1ee0blyr29ibbmz2ks*2'
        }).content
    except:
        content = ''
    return content


def save_2_db(save_data, db, collection):
    if not save_data or not db or not collection:
        return 0
    coll = CONN[db][collection]
    if isinstance(save_data, dict):
        return _save_2_db_single(save_data, coll)
    if isinstance(save_data, list):
        saved = 0
        for data in save_data:
            saved += _save_2_db_single(data, coll)
        return saved
    return 0


def _save_2_db_single(save_data, coll):
    try:
        is_exist = coll.find_one({'_id': save_data['_id']})
        save_data['update_time'] = datetime.datetime.utcnow()
        if not is_exist:
            coll.insert(save_data)
            ret = 1
        else:
            coll.update({'_id': save_data['_id']}, {'$set': dict(is_exist, **save_data)}, True)
            ret = 1
    except Exception, e:
        logger.info('save single into db error : %s.' % e)
        ret = 0
    return ret


def deal_str(string):
    if string:
        return string.replace('\n', '').replace(' ', '').replace('\t', '').replace('\r', '')
    return ''


def select_from_db(select_data, db, collection, count=10000):
    result_list = []
    if not select_data or not db or not collection or not isinstance(select_data, dict):
        return result_list
    coll = CONN[db][collection]
    for index, data in enumerate(coll.find(select_data)):
        if index >= count:
            break
        result_list.append(data)
    return result_list


if __name__ == '__main__':
    pass